load('./temp/losses')
load('./temp/probes')

load('./temp/config')
load('./temp/classifier_bank')

n_dataset = length( config.datasets );
classifier_family = cell( 1, n_dataset );

%for demo specific
classifier_family{1} = [ 2 ];
classifier_family{2} = [ 1 ];

n_classifier = length( classifier_bank_path );

clear classifier_list

for i = 1:n_dataset
    classifier_list{i} = [];
    
    for j = 1:n_classifier
        n2 = length( config.datasets{i}.name );
        if length( classifier_bank_path(j).name) < n2
            continue;
        end
        
        if strcmp( classifier_bank_path(j).name(1:n2) , config.datasets{i}.name )
            classifier_list{i} = [classifier_list{i} j];
        end               
    end
end


load('./temp/pairs')

addpath(genpath('./loss_estimate/'))
addpath(genpath('./../util/'))

for i = 1:size(losses,1)
    n = size( losses, 2);
    m = size( losses{i,1} ,2);
    
    temp = zeros( m , n );
    
    for j = 1 : n 
        temp( :, j )  = losses{i , j};
%         imagesc(temp);
%         if(mod(j,10)==0),drawnow;end
    end    
    
    loss{i} = temp;
    
    n = size( probes, 2);
    
    temp = zeros( m, n );
    
    for j = 1:n
        temp( : , j) = probes{ i , j };
%         imagesc(temp);
%         if(mod(j,10)==0),drawnow;end
    end
    
    probe{i} = temp;    
end

train_method = @(feat,res)(KNNTrain(feat,res,8));
test_method  = @KNNPredict;


config.loss_estimate_train = train_method;
config.loss_estimate_predict = test_method;

config.appear_probe_method = @(img)getHist(img);

config.using_appear_probe = 1;

report_save_name = ['./temp/demo_report'];

%load('./temp/appear_probe.mat');

n_probe = size( pairs, 1);

probe_method = @(img)getHist(img);

appear_probe = getAppearProbes( config , probe_method );

%%

for dataset_i = 1:2
    disp( config.datasets{dataset_i}.name )
    %evaluate on each dataset

    %prepare probe and losses for this dataset
    classifier_sel_ord = [];

    %probe_sel_ord;

    for j = classifier_family{dataset_i}
        classifier_sel_ord = [classifier_sel_ord classifier_list{j}];
    end

    loss_test = loss{dataset_i}(:,classifier_sel_ord);    

    loss_train = [];    

    for j = classifier_family{dataset_i}
        loss_train = [loss_train; loss{j}(:,classifier_sel_ord)]; 
    end

    m = size( loss_train, 2);
    
    for k = 1:size(loss_train,1)
        [val,id] = sort(loss_train(k,:));        
    end



    bo = zeros( 1, n_classifier);
    bo( classifier_sel_ord ) = 1;
    probe_sel_ord = [];

    for j = 1:n_probe
        if( bo( pairs( j , 1 ) ) &&  bo( pairs( j , 2 ) ) )
            probe_sel_ord = [probe_sel_ord j];
        end
    end



    probe_test = probe{dataset_i}(:, probe_sel_ord );
    probe_train = [];
    for j = classifier_family{dataset_i}
        probe_train = [probe_train; probe{j}(:,probe_sel_ord)]; 
    end

    if config.using_appear_probe
        temp = [];
        for j = classifier_family{dataset_i}
            temp = [temp; appear_probe{j} ];
        end
        probe_train = [probe_train temp];

        probe_test = [probe_test appear_probe{dataset_i}];
    end





    loss_estimate_model = config.loss_estimate_train( probe_train, loss_train );

    loss_test_est = config.loss_estimate_predict(loss_estimate_model, probe_test );

    res = getKPrecision( loss_test_est, loss_test);

    recommendation_report{dataset_i}.loss_test_est = loss_test_est;
    recommendation_report{dataset_i}.loss_test = loss_test;
    recommendation_report{dataset_i}.K_Precision_Curve = res;
    recommendation_report{dataset_i}.classifier_id = classifier_sel_ord;    
    
    clear loss_test_est
    clear loss_test
    clear res
    clear classifier_sel_ord
    clear loss_estimate_model            
end

save(report_save_name,'recommendation_report');

%%

n_dataset = length( config.datasets );

config.res_type = 'jpg';

Voting_Method = 'Uniform';

load( report_save_name );

%%

for dataset_i = n_dataset:n_dataset
    loss_test_est = recommendation_report{dataset_i}.loss_test_est;
    classifier_id = recommendation_report{dataset_i}.classifier_id;
    score_curve = evaluateVoting(config, dataset_i , loss_test_est, classifier_id , classifier_bank_path, Voting_Method, 5);
            
    F1_report{dataset_i} = score_curve;                        
end